{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ec45199a-0beb-4307-bd48-80683fd1084b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "from lxml import etree\n",
    "import requests\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1959b698-929b-454f-a426-766c47e0ad95",
   "metadata": {},
   "outputs": [],
   "source": [
    "##请求网站信息\n",
    "def req(url):\n",
    "    headers = {\n",
    "        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.46',\n",
    "        'Accept-Encoding': 'gzip, deflate',\n",
    "        'referer': url,\n",
    "        'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2', \n",
    "    }\n",
    "    time.sleep(random.random()*6)\n",
    "    response = requests.get(url,headers=headers) ##加请求头\n",
    "    if response.status_code == 200:\n",
    "        return response\n",
    "    else:\n",
    "        raise EOFError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "20ec265e-bffd-4fe8-bd17-546e25d6b3d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#解析网站\n",
    "def crawl(url):\n",
    "    res = req(url)\n",
    "    configs = []\n",
    "    html = etree.HTML(res.text)\n",
    "    title = html.xpath('/html/body/div[1]/div/div/section/section[3]/section[1]/section[2]//div/a/div[2]/div[1]/div[1]/h3/text()')\n",
    "    config = html.xpath('/html/body/div[1]/div/div/section/section[3]/section[1]/section[2]//div/a/div[2]/div[1]/section/div/p[1]/span/text()')\n",
    "    area = html.xpath('/html/body/div[1]/div/div/section/section[3]/section[1]/section[2]//div/a/div[2]/div[1]/section/div/p[2]/text()')\n",
    "    direction = html.xpath('/html/body/div[1]/div/div/section/section[3]/section[1]/section[2]//div/a/div[2]/div[1]/section/div/p[3]/text()')\n",
    "    floor = html.xpath('/html/body/div[1]/div/div/section/section[3]/section[1]/section[2]//div/a/div[2]/div[1]/section/div/p[4]/text()')\n",
    "    build_year =  html.xpath('/html/body/div[1]/div/div/section/section[3]/section[1]/section[2]//div/a/div[2]/div[1]/section/div/p[5]/text()')\n",
    "    price =  html.xpath('/html/body/div[1]/div/div/section/section[3]/section[1]/section[2]//div/a/div[2]/div/p[1]/span[1]/text()')\n",
    "    config_len = len(config)\n",
    "    for j in range(0,config_len,6):\n",
    "        configs.append(\"\".join(config[j:j+6]).strip().replace(\"\\n\",\"\"))\n",
    "    return zip(title, configs,area,direction, floor, build_year, price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0e9b57b9-57fa-41d6-be4b-b74e4988282d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#清洗数据,因为是用xpath提取的，所以要将无关的符号去掉\n",
    "def details(url):\n",
    "    result = crawl(url)\n",
    "    data = []\n",
    "    for i in result:\n",
    "        data1 = []\n",
    "        for j in i:\n",
    "            data1.append(j.strip().replace(\"\\n\",\"\"))\n",
    "        data.append(data1)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "271402af-a885-40fc-ba57-b16e12991a9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#转换为df格式\n",
    "def get_df(url):\n",
    "    data = details(url)\n",
    "    df = pd.DataFrame(data)\n",
    "    df.columns = [\"title\",\"configs\",\"area\",\"direction\", \"floor\", \"build_year\", \"price\"]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6a9c278d-5a89-47d6-8ddc-fd84c5db4828",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 合并并存入csv\n",
    "def main(region):\n",
    "    if os.path.exists(f\"{region}.csv\"):\n",
    "        df_all = pd.read_csv(f\"{region}.csv\").iloc[:,1:]\n",
    "    else:\n",
    "        df_all = pd.DataFrame()\n",
    "    for n in tqdm([str(i) for i in range(0,15)]):\n",
    "        if int(n) % 3 == 0:\n",
    "            time.sleep(random.randint(13,20))\n",
    "        url = f\"https://shenzhen.anjuke.com/sale/{region}/p{n}/\"  # wuchang-11\n",
    "        try:\n",
    "            df = get_df(url)\n",
    "        except ValueError:\n",
    "            time.sleep(31)\n",
    "            try:\n",
    "                df = get_df(url)\n",
    "            except ValueError:\n",
    "                time.sleep(48)\n",
    "                try:\n",
    "                    df = get_df(url)\n",
    "                except ValueError:\n",
    "                    time.sleep(88)\n",
    "                    try:\n",
    "                        df = get_df(url)\n",
    "                    except ValueError:\n",
    "                        print(\"当前爬取失败的页数为：\",n)\n",
    "                        df_all.columns = [\"title\",\"configs\",\"area\",\"direction\", \"floor\", \"build_year\", \"price\"]\n",
    "                        df_all.to_csv(f\"{region}.csv\")\n",
    "                        raise EOFError\n",
    "        df_all = df_all.append(df)\n",
    "    df_all.columns = [\"title\",\"configs\",\"area\",\"direction\", \"floor\", \"build_year\", \"price\"]\n",
    "    df_all.to_csv(f\"{region}.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "da86af3b-b031-462a-a10a-6843bdd18e1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                           | 0/15 [00:00<?, ?it/s]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      "  7%|█████▌                                                                             | 1/15 [00:20<04:47, 20.55s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      " 13%|███████████                                                                        | 2/15 [00:22<02:03,  9.50s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      " 20%|████████████████▌                                                                  | 3/15 [00:24<01:13,  6.14s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      " 27%|██████████████████████▏                                                            | 4/15 [00:48<02:23, 13.03s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      " 33%|███████████████████████████▋                                                       | 5/15 [00:49<01:30,  9.01s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      " 40%|█████████████████████████████████▏                                                 | 6/15 [00:54<01:07,  7.46s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      " 47%|██████████████████████████████████████▋                                            | 7/15 [01:19<01:44, 13.08s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      " 53%|████████████████████████████████████████████▎                                      | 8/15 [01:25<01:17, 11.02s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      " 60%|█████████████████████████████████████████████████▊                                 | 9/15 [01:31<00:55,  9.26s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      " 67%|██████████████████████████████████████████████████████▋                           | 10/15 [01:49<01:00, 12.15s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      " 73%|████████████████████████████████████████████████████████████▏                     | 11/15 [01:55<00:41, 10.34s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      " 80%|█████████████████████████████████████████████████████████████████▌                | 12/15 [01:57<00:22,  7.61s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      " 87%|███████████████████████████████████████████████████████████████████████           | 13/15 [02:11<00:19,  9.63s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      " 93%|████████████████████████████████████████████████████████████████████████████▌     | 14/15 [02:13<00:07,  7.26s/it]C:\\Users\\lucky\\AppData\\Local\\Temp\\ipykernel_75296\\26652539.py:30: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_all = df_all.append(df)\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████| 15/15 [02:18<00:00,  9.21s/it]\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    main(region=\"yantian\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c97b9db5-3107-4589-b8c1-b4ec5ebd3908",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
